\documentclass{beamer}
\usepackage[latin1]{inputenc}
\usepackage{amsthm}
\usepackage{tikz}
\usepackage{graphicx}
\usepackage{multirow}
\usepackage{hanging}
\usepackage{bibentry}
\usetheme{Luebeck}
\usecolortheme{beaver}
\useoutertheme{split}
\title{The Problems of Predicting Participation in Energy Efficiency Programs}
\author{Alexander L. Davis \\ Tamar Krishnamurti}
\date{\today}
\institute{Department of Engineering and Public Policy \\ Center for Climate and Energy Decision Making \\ Carnegie Mellon University}

\setbeamertemplate{frametitle}
                  {\begin{centering}\smallskip
\insertframetitle\par
   \smallskip\end{centering}}
\setbeamertemplate{itemize item}{$\bullet$}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{footline}[text line]{%
  \hfill\strut{% 
    \scriptsize\sf\color{black!60}%
    \quad\insertframenumber
    }%
    \hfill
}

\makeatletter
\def\beamer@autobreakframebox{%
  \global\setbox\beamer@splitbox=\box\voidb@x%
  \ifbeamer@autobreak%
    % Ok, frame was overful -> split it!
    \setbox\@tempboxa=\vsplit\beamer@framebox to\beamer@autobreakfactor\textheight%
    \global\setbox\beamer@splitbox=\box\beamer@framebox%
    \@tempdima=\ht\beamer@splitbox%
    \ifdim\@tempdima<\beamer@autobreaklastheight%
      \global\beamer@autobreaklastheight=\@tempdima\relax%
    \else%
      \setbox\@tempboxa=\vbox{\unvbox\@tempboxa\unvbox\beamer@splitbox}%
      \global\setbox\beamer@splitbox=\box\voidb@x%
    \fi%
    \setbox\beamer@framebox=\vbox to\textheight{\unvbox\@tempboxa%
      \vskip\beamer@framebottomskipautobreak%
      \ifvoid\beamer@splitbox%
        \ifvoid\beamer@footins%
        \else%
          \begingroup
            \usebeamercolor*[fg]{footnote}%
%            \footnoterule%
            \unvbox \beamer@footins%
            \global\setbox\beamer@footins=\box\voidb@x%
          \endgroup  
        \fi%
      \fi%
      \beamer@exitcode%
    }%
  \else%
    \setbox\beamer@framebox=\vbox to\textheight{\unvbox\beamer@framebox%
      \vskip\beamer@framebottomskip%
      \ifvoid\beamer@footins%
      \else%
        \begingroup
          \usebeamercolor*[fg]{footnote}%
%         \footnoterule%
          \unvbox \beamer@footins%
          \global\setbox\beamer@footins=\box\voidb@x%
        \endgroup 
      \fi%
      \beamer@exitcode}%
    \global\setbox\beamer@footins=\box\voidb@x%
  \fi%
  }
\makeatother

\setbeamertemplate{footnote}{%
  \hangpara{0.1em}{0}%
  \makebox[0.6em][l]{\tiny\insertfootnotemark}\tiny\insertfootnotetext\par%
}

\newenvironment<>{varblock}[2][\textwidth]{% 
  \setlength{\textwidth}{#1} 
  \begin{actionenv}#3% 
  \def\insertblocktitle{\centering#2\par}% 
  \par% 
  \usebeamertemplate{block begin}} 
{\par% 
  \usebeamertemplate{block end}% 
  \end{actionenv}}

\renewcommand\footnotemark{}
\renewcommand\footnoterule{}

\tikzset{
  every overlay node/.style={
    draw=black,fill=white,rounded corners,anchor=north west,
  },
}
% Usage:
% \tikzoverlay at (-1cm,-5cm) {content};
% or
% \tikzoverlay[text width=5cm] at (-1cm,-5cm) {content};
\def\tikzoverlay{%
   \tikz[baseline,overlay]\node[every overlay node]
}%

\mode<presentation>
% Define some colors:
%\definecolor{chocolate}{RGB}{33,33,33}
\definecolor{darkred}{rgb}{0.8,0,0}
\definecolor{darkblue}{rgb}{0,0,0.8}

% Set some colors:
\setbeamertemplate{caption}{\insertcaption} 
\setbeamertemplate{caption label separator}{}
\setbeamercolor{structure}{bg=gray,fg=gray}
\setbeamercolor{block title}{bg=gray!30!white,bg=darkred}
\setbeamercolor{title}{bg=gray!30!white,fg=darkred}
%\setbeamercolor{block body}{bg=normal text.bg!90!black}
\setbeamercolor{section in toc}{fg=darkred}
\setbeamercolor{itemize item}{fg=darkred}
\setbeamercolor{enumerate item}{fg=darkred}
\setbeamercolor{section in toc}{fg=black,bg=white}
\setbeamercolor{alerted text}{fg=darkred!80!gray}
\setbeamercolor*{palette primary}{fg=darkred!60!black,bg=gray!30!white}
\setbeamercolor*{palette secondary}{fg=darkred!70!black,bg=gray!15!white}
\setbeamercolor*{palette tertiary}{bg=darkred!80!black,fg=gray!10!white}
\setbeamercolor*{palette quaternary}{fg=darkred,bg=gray!5!white}
\setbeamercolor*{sidebar}{fg=darkred,bg=gray!15!white}
\setbeamercolor*{palette sidebar primary}{fg=darkred!10!black}
\setbeamercolor*{palette sidebar secondary}{fg=white}
\setbeamercolor*{palette sidebar tertiary}{fg=darkred!50!black}
\setbeamercolor*{palette sidebar quaternary}{fg=gray!10!white}
\setbeamercolor{titlelike}{parent=palette primary,fg=black}
\setbeamercolor{frametitle}{bg=gray!10!white}
\setbeamercolor{frametitle right}{bg=gray!60!white}
\setbeamercolor*{separation line}{}
\setbeamercolor*{fine separation line}{}
\mode
<all>

\nobibliography*

\begin{document}
\maketitle

\section{Problem}
\subsection{Definition}
\begin{frame}[noframenumbering]
  \frametitle{Volunteer Bias} \pause
  \begin{varblock}{Definition} \pause
    A volunteer sample provides a biased estimate of the treatment effect in a population. \pause
   \end{varblock}
\begin{itemize}
\item Most program evaluations depend on all-volunteer samples, usually early adopters.
\end{itemize}
\end{frame}

\subsection{My Reasons}
\begin{frame} 
  \frametitle{Why Volunteer Bias?} \pause
  \begin{varblock}{My Reasons} \pause
    \begin{itemize}
    \item Pervasive problem without a solution or advocate.\pause
    \item Widely applicable: energy, medicine, education, etc. \pause
    \item Subtle problem that many won't recognize. \pause
    \item Deceptive, allowing the public to be fleeced.
      \begin{itemize}
      \item (this really annoys me)
      \end{itemize}
    \end{itemize}
  \end{varblock}
\end{frame}

\subsection{Why you should care}
\begin{frame} 
  \frametitle{Effect on you} \pause
  \begin{varblock}{Researchers} \pause
    \begin{itemize}
    \item Exacerbates the challenge of recruitment in your behavioral studies. \pause  
    \end{itemize}
  \end{varblock}
  \begin{varblock}{Decision-Makers} \pause
    \begin{itemize}
    \item Cost-benefit estimates will be skewed toward favorability. \pause
    \item Unwarranted inferences from studies with biased samples can waste money and delay the discovery of effective alternatives. \pause
    \item PUC, DOE, and Utility may endorse, subsidise, or provide energy efficiency programs and technologies that are ineffective.
    \end{itemize}
  \end{varblock}
\end{frame}

\section{In-Home Displays}
\subsection{In-Home Displays}
\begin{frame}[plain]
  \frametitle{Example: In-Home Displays} \pause
\begin{figure} 
\centering
  \includegraphics[height=7cm]{largePCM.jpg} \pause 
 \end{figure}
\begin{itemize}
  \item Primary Goal: Reduce overall household energy consumption. \pause
    \item Secondary Goal: Promote learning about causes of use.
      \end{itemize}
\end{frame}

\subsection{Volunteer Bias}
\begin{frame} 
  \frametitle{Volunteer Bias} \pause
  \begin{varblock}{Prior Evidence} \pause
    \begin{itemize}
    \item 17 studies in the US/Canada evaluated 11 different displays.  \pause
    \item 13 studies included only volunteers (76\%). \pause
    \item 12 recruited treatment and control group differently (70\%). \pause
      \begin{itemize}
        \item This is really, really, really-really-really, bad. \pause
      \end{itemize}
    \item 4 had low risk of volunteer bias (24\%). \pause
      \begin{itemize}
      \item 1 study had no opt-outs. \pause
      \item 1 study participants didn't know about the study. \pause
      \item 2 studies used propensity score adjustment.
      \end{itemize}           
    \end{itemize}
  \end{varblock}
\end{frame}

\begin{frame} 
  \frametitle{Risks} \pause
  \begin{varblock}{Policy} \pause
    \begin{itemize}
    \item PUC mandates an ``energy speedometer.'' \pause 
    \item Utility may buy technology, passing costs to customers. \pause
    \end{itemize}
\end{varblock}
  \begin{varblock}{Industry} \pause
    {\footnotesize ``Hydro One, Ontario's largest utility, indicated that the presence of real time feedback devices has a measurable and positive impact on energy conservation.''} \pause {\footnotesize ``Save up to 20\% off your electricity bill!''} \pause 
    \begin{itemize}[]
    \item Actual \emph{unadjusted} overall conservation was 6.5\%. \pause
    \item This study had every bias we assessed in our meta-analysis:\footnote<9->{\bibentry{davis2012setting}} \pause
      \begin{itemize}
      \item Volunteers, Lack of randomization, Lack of blinding, Attrition
      \end{itemize}
    \end{itemize}
  \end{varblock}
\end{frame}

\subsection{Solutions}
\begin{frame} 
  \frametitle{Solutions} \pause
  \begin{varblock}{Study Design} \pause 
    \begin{itemize}
    \item Randomization after recruitment. \pause
    \item Opt-out, as non-responders are more procrastinators/misinformed than rejectors.\footnote<4->{\bibentry{williams2007no}} \pause    
    \item Recruitment best practices (can be kept under 30\%). \pause  
    \end{itemize}
  \end{varblock}
  \begin{varblock}{Adjustment} \pause
    \begin{itemize}
    \item Sample non-respondents. \pause
    \item Extrapolate based on temporal information.\footnote<8->{\bibentry{armstrong1977estimating}}
    \end{itemize}
  \end{varblock}
\end{frame}

%hewison2006overcoming,campbell1949bias,,junghans2005recruiting

\section{Propensity Score}
\begin{frame}
  \frametitle{Propensity Score} \pause
\begin{varblock}{Propensity Score} \pause
  \begin{itemize}
    \item If we can confidently predict who will volunteer, then average treatment effects will be unbiased.
  \end{itemize}
\end{varblock}
\end{frame}

\begin{frame} 
  \frametitle{Requirements of the model} \pause
  \begin{varblock}{Model Requirements} \pause
    \begin{itemize}
    \item Input variables to accurately predict enrollment (ignorability). \pause
    \item Estimation method that discovers the right configuration. \pause
    \item Recruit non-volunteers to train the model.
    \end{itemize}
  \end{varblock}
\end{frame}

\subsection{Inputs}
\begin{frame} 
  \frametitle{Model Inputs} \pause
  \begin{varblock}{Model Inputs} \pause
    \begin{itemize}
    \item Previous propensity score models.  \pause
    \begin{itemize}
      \item PG\&E's Smart-Rate Pilot.\footnote<4->{\bibentry{george2010load}} \pause
      \item Commonwealth Edison Energy Smart Pricing Pilot.\footnote<5->{\bibentry{sbc2007evaluation}} \pause
        \begin{itemize}
        \item Purchase of new major appliances, using fans to reduce costs, age, single-family detached home, \# and type of people in household. \pause
        \end{itemize}
      \end{itemize}
  \item Prior research on volunteering (survey research, psychology, medicine).\footnote<7->{\bibentry{rosenthal1975volunteer}} \pause  
  \item Those generated from open-ended, mental-models interviews. \pause
  \item Our expert judgment given the context. 
    \end{itemize}
  \end{varblock}
\end{frame}

\begin{frame}
  \frametitle{Survey} \pause
  we conducted a survey
  \end{frame}

\begin{frame}
  \frametitle{Prediction Task} \pause 
  \begin{varblock}{Your Prediction} \pause
    ``You have been chosen to receive a free in-home display. On the display you can see your home electricity use for one year. You will evaluate the display for 1 year. At the end of the year, you can keep it if you like it or return it for a \$25 gift certificate. We will send you four short surveys (one every 3 months) over the year asking how useful you find the display and about your electricity use. If you choose to participate, you will receive the display 3 weeks from now. There will be no cost to you and your information and survey responses will be confidential, as is university policy.'' \pause
  \end{varblock}
  \begin{itemize}
  \item What do you think will predict volunteering? (write down).
    \end{itemize}
\end{frame}

\begin{frame} 
  \frametitle{Demographics} \pause
  \begin{varblock}{Will demographics predict?}
Age, Gender, Employment Status, Education, Annual Household Income, Race, Political Affiliation, Adults/Children in the home.
  \end{varblock}
  \vspace{8pt}
  \centering
      {\huge Yes or No?} \\ \pause
      \vspace{8pt}
      \begin{center}
             {\Huge No} \pause
             \end{center}
\begin{center}
  These are standard demographics, some of which have been shown to predict volunteering previously.
\end{center}
\end{frame}

\begin{frame} 
  \frametitle{Number of hours in the home} \pause
  \begin{varblock}{Will number of hours in the home predict?} 
6am-10am, 10am-2pm, 2pm-6pm, 6pm-10pm, 10pm-2am, 2am-6pm
  \end{varblock}
  \vspace{8pt}
  \centering
      {\huge Yes or No?} \\ \pause
      \vspace{8pt}
      \begin{center}
             {\Huge Yes} \pause
             \end{center}
\begin{center}
Participants need to be in the home to expect that the trial would be worthwhile. \\
\end{center}    
\end{frame}

\begin{frame} 
  \frametitle{Trust} \pause
  \begin{varblock}{Will trust predict?} 
\begin{itemize}
  \item Local government
  \item Scientists
  \item Utility Company
\end{itemize}
  \end{varblock}
  \vspace{8pt}
  \centering
      {\huge Yes or No?} \\ \pause
      \vspace{8pt}
      \begin{center}
             {\Huge Yes} \pause
      \end{center}
\begin{center}
  Survey literature suggests trust in the source matters. 
\\ \vspace{3pt}
  {\scriptsize ($\mathbf{\tau=.14}$, Factor Loading $= 53\%$, Cronbach's $\alpha=$.78)}
\end{center}    
\end{frame}

\begin{frame} 
  \frametitle{Self-Efficacy} \pause
  \begin{varblock}{Will self-efficacy predict?} 
    \begin{itemize}
    \item ``If something looks too complicated I will not even bother to try it.''
    \item ``When I decide to do something, I go right to work on it.''  
    \end{itemize}
  \end{varblock}
  \vspace{8pt}
  \centering
      {\huge Yes or No?} \\ \pause
      \vspace{8pt}
      \begin{center}
             {\Huge No} \pause
             \end{center}
\begin{center}
The General Self-Efficacy Scale\footnote<4->{\bibentry{sherer1982self}} \\ \vspace{3pt}
  {\scriptsize ($\tau=.07$, Factor Loading $= 52\%$, Cronbach's $\alpha=$.74)}
\end{center}
\end{frame}

\begin{frame} 
  \frametitle{Social Networks} \pause
  \begin{varblock}{Will integration in social networks predict?} 
    \begin{itemize}
  \item ``How many close friends do you have?''
  \item ``How many of these friends do you see or talk to at least once every 2 weeks?''
\end{itemize}
  \end{varblock}
  \vspace{8pt}
  \centering
      {\huge Yes or No?} \\ \pause
      \vspace{8pt}
      \begin{center}
             {\Huge No} \pause
             \end{center}
\begin{center}
  The Social Network Index\footnote<4->{\bibentry{cohen1997social}}
\\ \vspace{3pt}
 {\scriptsize ($\tau=.06$, Factor Loading $= 73\%$, Cronbach's $\alpha=$.81)}
\end{center}
\end{frame}

\begin{frame} 
  \frametitle{Frugality} \pause
  \begin{varblock}{Will frugality predict?} 
    \begin{itemize}
  \item ``There are many things that are normally thrown away that are still quite useful.''
\end{itemize}
  \end{varblock}
  \vspace{8pt}
  \centering
      {\huge Yes or No?} \\ \pause
      \vspace{8pt}
      \begin{center}
             {\Huge No} \pause
             \end{center}
      \begin{center}
          Lastovicka Frugality Scale\footnote<4->{\bibentry{lastovicka1999lifestyle}}
\\ \vspace{3pt}
  {\scriptsize ($\tau=.02$, Factor Loading $= 58\%$, Cronbach's $\alpha=$.82)}
      \end{center}
\end{frame}

\begin{frame} 
  \frametitle{Exploration and Curiosity} \pause
  \begin{varblock}{Will exploration and curiosity predict?} 
\begin{itemize}
  \item ``I like to read books by writers I've not come across before.''
  \item ``I like to try to solve problems that present a mental challenge.''    
\end{itemize}
  \end{varblock}
  \vspace{8pt}
  \centering
      {\huge Yes or No?} \\ \pause
      \vspace{8pt}
      \begin{center}
             {\Huge Yes} \pause
             \end{center}
\begin{center}
Curiosity Index\footnote<4->{\bibentry{derek1974construct}}
\\ \vspace{3pt}
  {\scriptsize ($\mathbf{\tau=.11}$, Factor Loading $= 63\%$, Cronbach's $\alpha=$.92)}
\end{center}
\end{frame}

\begin{frame} 
  \frametitle{Concern for the Environment} \pause
  \begin{varblock}{Will concern for the environment predict?} 
\begin{itemize}
  \item ``The balance of nature is very delicate.''
  \item ``Plants and animals exist primarily to be used by humans.''
\end{itemize}
  \end{varblock}
  \vspace{8pt}
  \centering
      {\huge Yes or No?} \\ \pause
      \vspace{8pt}
      \begin{center}
             {\Huge No} \pause
             \end{center}
\begin{center}
  The New Ecological Paradigm\footnote<4->{\bibentry{dunlap2000new}}
\\ \vspace{3pt}
  {\scriptsize ($\tau=.08$, Factor Loading $= 48\%$, Cronbach's $\alpha=$.73)}
\end{center}
\end{frame}

\begin{frame} 
  \frametitle{Expectations of the In-home display} \pause
  \begin{varblock}{Will expectations of the in-home display predict?} 
\begin{itemize}
  \item ``I expect to save money using the in-home display.''
  \item ``I expect to enjoy using the in-home display.''
\end{itemize}
  \end{varblock}
  \vspace{8pt}
  \centering
      {\huge Yes or No?} \\ \pause
      \vspace{8pt}
      \begin{center}
             {\Huge Yes} \pause
             \end{center}
\begin{center}
Our intuition that expected benefit from the specific offering is a strong motivator of volunteering.
\\ \vspace{3pt}
  {\scriptsize ($\mathbf{\tau=.35}$, Factor Loading $= 78\%$, Cronbach's $\alpha=$.93)}
\end{center}
\end{frame}

\begin{frame} 
  \frametitle{Making Eco-Friendly Purchases} \pause
  \begin{varblock}{Will the desire to make eco-friendly purchases predict?} 
\begin{itemize}
  \item ``I have switched products for ecological reasons.''
  \item ``I have purchased a household appliance because it uses less electricity than other brands.''
\end{itemize}
  \end{varblock}
  \vspace{8pt}
  \centering
      {\huge Yes or No?} \\ \pause
      \vspace{8pt}
      \begin{center}
             {\Huge No} \pause
             \end{center}
\begin{center}
  Ecological Purchasing\footnote<4->{\bibentry{fraj2006ecological}}
\\ \vspace{3pt}
  {\scriptsize ($\tau=.03$, Factor Loading $= 59\%$, Cronbach's $\alpha=$.78)}
\end{center}  
\end{frame}

\begin{frame} 
  \frametitle{Environmental Social Comparisons} \pause
  \begin{varblock}{Will environmental social comparisons predict?} 
\begin{itemize}
 \item ``My household has done more to reduce its electricity consumption.''
 \item ``My household cares more about the environment.''
\end{itemize}
  \end{varblock}
  \vspace{8pt}
  \centering
      {\huge Yes or No?} \\ \pause
      \vspace{8pt}
      \begin{center}
             {\Huge No} \pause
             \end{center}
\begin{center}
  Social norms and social behavior\footnote<4->{\bibentry{reno1993transsituational}}
  \end{center}      
\end{frame}

\begin{frame} 
  \frametitle{Items Generated from Mental Models Interviews} \pause
  \begin{varblock}{Will items generated from mental models interviews predict?} 
\begin{itemize}
  \item ``I would participate to promote energy independence in the US.''
  \item ``I would participate to avoid wasting energy.''
\end{itemize}
  \end{varblock}
  \vspace{8pt}
  \centering
      {\huge Yes or No?} \\ \pause
      \vspace{8pt}
      \begin{center}
             {\Huge No} \pause
             \end{center}
\begin{center}
  These predicted acceptance of smart-meters well, but not volunteering intentions. 
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{Evaluation of Inputs}
  \begin{itemize}
  \item we took stuff from previous propensity models in this area; we took stuff from volunteeringliteratures across various disciplines; we asked people in mental models; we brainstormed and use expert judgment; the hit rate was low
  \item talk about how unintuitive this stuff is; even for the person making the decision, they have difficulty with insight into what makes them work
  \item even when we ask people in mental models, they can't figure it out
  \item we can't rely on expert judgment, people's own theories, etc.
  \end{itemize}
\end{frame}
        
\subsection{Prediction}
\begin{frame} 
  \frametitle{Accuracy} \pause
\begin{varblock}{Univariate Prediction} \pause
  \begin{itemize}
  \item 15/72 individual items successful (21\%). \pause
  \item 4/12 constructs successful (PCA).\footnote<4->{\bibentry{loehlin2004latent}} \pause
  \item Best predictor: 12-17\% generalization error.\footnote<5->{Error rate estimated w/ 10-fold CV and bootstrap. \\ \vspace{5pt} \bibentry{efron1993introduction}} \pause
    \begin{itemize}
    \item Whether they thought they would enjoy the display.
      \end{itemize}
  \end{itemize}
  \end{varblock}
\end{frame}

\begin{frame} 
  \frametitle{Accuracy} \pause
\begin{varblock}{Logistic Regression} \pause
  \begin{itemize}
  \item 4/72 items used in the model (1/12 constructs).  \pause
    \begin{itemize}
    \item Expected enjoyment of the in-home display. \pause 
    \item Expected learning from the in-home display. \pause
    \item Whether they trust the utility. \pause
    \item Whether they are active in their community. \pause
    \end{itemize}
  \item 15\% generalization error.    
  \end{itemize}
\end{varblock}
\end{frame}

\begin{frame} 
  \frametitle{Accuracy} \pause
  \begin{varblock}{Classification Tree} \pause
    \begin{itemize}
    \item 5/72 items used in the model. \pause
      \begin{itemize}
      \item Expected enjoyment of the in-home display. \pause
      \item In the home from 6am-10am. \pause
      \item Whether they trust scientists. \pause
      \item Whether they trust friends. \pause
      \item Whether they handle unexpected problems well. \pause
      \end{itemize}
    \item 10-13\% generalization error. 
    \end{itemize}
  \end{varblock}
\end{frame}

%transition

\subsection{Recruiting non-volunteers}
\begin{frame} 
  \frametitle{Recruiting non-volunteers} \pause
\begin{varblock}{Recruiting non-volunteers} \pause
  \begin{itemize}
      \item All participants receive a 5-item survey with their recruitment offer. \pause
      \item Send two postcard follow-ups with at most 5 items. \pause
      \item Sample non-responders for interviews.\footnote<5->{\bibentry{clausen1947controlling}}
  \end{itemize}
  \end{varblock}
\end{frame}

%kanuk1975mail,rosenthal1975volunteer,ferber1948problem,ford1949bias,campbell1949bias}.

\subsection{Checklist}

%transition; comes out of nowhere

\begin{frame}
  \frametitle{Checklist} \pause
  \begin{varblock}{Checklist} \pause
    Checklists have been very effective in promoting correct completion of routinized but complicated actions.\footnote<3->{\bibentry{gawande2010checklist}}
  \end{varblock}
\end{frame}

\begin{frame} 
  \frametitle{Checklist} \pause
\begin{center}
\scalebox{0.55}{
  \begin{tabular}{p{12cm} |c| |c|}
Item & Yes & No \\ \hline \pause
\underline{Recruitment} & & \\ 
Have recruitment best practices been used? & & \\
Has opt-out been used? & & \\
Have 2 follow-up post-cards/letters been used? & & \\
Do these postcards include 1-5 questions about non-response? & & \\
Has a small up-front incentive (\$2) been used? & & \\ \pause
& & \\
\underline{Inputs} & & \\ 
Have inputs been derived from volunteering research in general? & & \\
Do inputs include constraints, topical interest, expected benefit, and exploratory personality? & & \\
Do inputs include lay theories about volunteering in this context? & & \\ \pause
& & \\
\underline{Estimation} & & \\ 
Has a classification tree been used to discover the necessary questions? & & \\
Has the best model been selected based on generalization error (CV and Bootstrap)? & & \\ \pause
& & \\
\underline{Sampling Non-Responders} & & \\
Do follow-up postcards include 1-5 questions about reasons for non-response? & & \\
Is the reason for non-response survey very short (less than 1 minute)? & & \\
Has a subsample of non-respondents been interviewed? & & \\ \hline
  \end{tabular}}
\end{center}
\end{frame}

\subsection{Summary}
\begin{frame}
  \frametitle{Summary} \pause
\begin{varblock}{Summary} \pause
  \begin{itemize}
  \item Volunteer bias is a serious and unsolved problem. \pause
  \item A propensity-type adjustment can yield unbiased results. \pause
  \item It is difficult to select the right inputs to the model, but so far... \pause
    \begin{itemize}
    \item Expectations of benefit specific to the program. \pause
    \item Constraints on participation (home during the day). \pause
    \item Trust in scientists. \pause
    \end{itemize}
    \item Classification Trees perform well, but not much better than the best predictor. \pause
    \item Short follow-up postcards and interviews may need only 1 question. 
    \end{itemize}
\end{varblock}
\end{frame}

\section{Future Directions}
\begin{frame} 
  \frametitle{Future Directions} \pause
  \begin{varblock}{Three Future Directions} \pause
    \begin{itemize}
    \item Improving theory and the psychology of prediction. \pause
    \item Modern alternative to propensity score. \pause
    \item Actual implementation in a real trial. 
    \end{itemize}
  \end{varblock}
\end{frame}

\subsection{Improving theory and the psychology of prediction}
\begin{frame} 
  \frametitle{Future Directions} \pause
  \begin{varblock}{Improving theory} \pause
    \begin{itemize}
    \item A review paper on the validity of different methods/predictors of volunteering is needed. \pause
    \item Theory is lacking, especially with respect to more basic psychological models. 
    \end{itemize}
    \end{varblock}
  \end{frame}

\begin{frame} 
  \frametitle{Future Directions} \pause
  \begin{varblock}{The psychology of prediction} \pause
\centering
    We've known this type of prediction is difficult for a while. \pause
      \begin{itemize}
      \item Self-knowledge is limited.\footnote<4->{\bibentry{nisbett1977telling}} \pause
      \item Other-knowledge is limited.\footnote<5->{\bibentry{ross1977intuitive}} \pause
      \item Complex models rarely better than simpler ones.\footnote<6->{\bibentry{dana2004superiority}} \pause
      \item Experts (us) do not perform better than laypeople.\footnote<7->{\bibentry{armstrong2012predictive}} \pause
        \begin{itemize}
          \item Advertising, Clinical Psychology, stock market forecasting \pause
        \end{itemize}            
      \item ``suckers will pay for the existence of seers''
      \end{itemize}
  \end{varblock}
 \end{frame}

%\footnote<10->{\bibentry{cowles1933can}}
%\footnote<9->{\bibentry{taft1955ability}} \footnote<11->{\bibentry{armstrong1980seer}}  
%chapman1969illusory,armstrong2011evidence,armstrong1991prediction,

\begin{frame} 
  \frametitle{Future Directions} \pause
  \begin{varblock}{A citizen-scientist alternative approach} \pause
    \begin{itemize}
    \item Mine lay intuition to complement the usual ``expert'' hypothetico-deductive approach. \pause
    \item This allows: \pause
      \begin{enumerate}
      \item Understanding the psychology of ``folk theories'' and prediction.\footnote<5->{\bibentry{keil2003folkscience}} \pause
      \item Possible discovery of the ``right theory'' by a layperson.\footnote<6->{\bibentry{khatib2011algorithm}} \pause
      \item Understanding two-way communication between scientists and citizen-scientists. 
      \end{enumerate}
      \end{itemize}
  \end{varblock}  
\end{frame}


\subsection{Modern alternatives to propensity score}
\begin{frame} 
  \frametitle{Future Directions} \pause
  \begin{varblock}{Modern estimation alternative: Semi-supervised learning} \pause
    \begin{itemize}
    \item Simple example: Self-training\footnote<3->{\bibentry{zhu2009introduction}} \pause
      \begin{enumerate}
      \item Estimate average treatment effect (ATE) based on predictors of volunteering. \pause
      \item Find nearest match among non-volunteers in the control and treatment group. \pause
      \item Add this participant with the estimate from the ATE model. \pause
      \item Repeat until treatment effect is estimated for all non-volunteers.
      \end{enumerate}
    \end{itemize}
  \end{varblock}
\end{frame}

\subsection{Actual implementation in a real trial}
\begin{frame} 
  \frametitle{Future Directions} \pause
\begin{varblock}{Conducting a real trial} \pause
  \begin{itemize}
    \item The approach needs to be validated with real data. \pause
    \item The approach has the potential to greatly improve the validity of any behavioral trial.
  \end{itemize}  
\end{varblock}
\end{frame}

\section{Sources}
\begin{frame}
  The contents of this talk, including data, analyses, and slides are available: \\ \vspace{5pt}
\begin{center}
  ``Predicting Volunteering for Energy Efficiency Programs,'' \url{http://hdl.handle.net/1902.1/19154} V2 [Version]
\end{center}
\end{frame}

\begin{frame}
{\Huge \centering References}
\end{frame}

\bibliographystyle{apalike}
\bibliography{/home/alex/Dropbox/masterbib}
\end{document}



    generating/ranking predictions 
   
The assumption is that experts (us) or laypeople have accurate knowledge into what works.
    \begin{itemize}
    \item Reasons for skepticism
      \begin{itemize}
      \end{itemize}
    \item Observables (electricity use); not so good \pause  
    \item Psychodemographics
    \item Unobtrusive measures 
    \item Asking people what would get them to volunteer (spontaneous; Weiner); people will spontaneously generate causes for their own and other's behavior
    \item Presenting them different options, predicting intentions
    \item Actual behavior
    \end{itemize}
  \end{varblock}  
\end{frame}

\begin{frame} 
  \frametitle{Theory: Successes, Failures and Extensions} \pause
Why is this problem hard?
a) People can assess frequency information, but it is absent
b) Causal models or intuitive theories must be used, that involve ontological committments (concepts) and bayes nets
This causal interpretation of covariation information colors learning
c) Attributions of human behavior are notoriously difficult (fundamental attribution error)
reasoning about intentional agents is different from inanimate objects \cite{saxe2005secret}; people assume events are cuased by intentional agent actions
High functioning autistics can do folk physics but not folk psychology \cite{baron1999mathematician}; folk psychology
d) academics and experts fail at predicting what advertising will work; laypeople don't do much better; If we sample enough laypoeple, maybe one of them will generate the right theory about the domain; make analogy to fold-it; this allows us to both understand the psychology of prediction (humans predicting the behavior of other humans), which is relevant for EPP (e.g., predicting demand), but we also solve the problem of volunteer bias;
\end{frame}

    \item Overlap (there must be levels of IVs at which volunteers and non-volunteers have the same score). \pause
    \begin{itemize}  
    \item Heckman's model assumes to multivariate normal errors (more for proofs). \pause
    \item The joint distribution of the volunteer and primary model can be estimated with semi-parametric or non-parametric alternatives (e.g., Kernels, Splines) \cite{newey1990semiparametric}. \pause
    \item Classification trees work well on our data \cite{breiman1984classification}. \pause
    \end{itemize}
    

  example with graphical model and equations
2SLS or Logistic
Generalization error, 
Classification Tree, semi-supervised learning
IHD example

